package services;

import helpers.StringHelper;
import helpers.UrlHelper;

import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.TreeMap;
import java.util.concurrent.Future;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import systems.ConfigSystem;

import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPMethod;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.appengine.api.urlfetch.URLFetchServiceFactory;

import models.DomainModel;
import models.config.ServiceConfigModel;

public class UpdateDomain {
    
    private URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
    
    private DomainModel domain = new DomainModel();
    
    private TreeMap<String, String> check = new TreeMap<String, String>();
    
    private Future fDomain = null;
    
    private Future fWhois = null;
    
    private Future fIpAddress = null;
    
    private Future fAlexa = null;
    
    private Future fGoogleRank = null;
    
    private Future fAgeDomain = null;
    
    private Future fValidator = null;
    
    private Future fGoogleService = null;
    
    private Future fDns = null;
    
    private Future fWhoisIp = null;
    
    private String content = "";
    
    private String alexaContent = "";
    
    private List<String> googleAnalyticSites = new ArrayList<String>();
    
    private List<String> googleAdsenceSites = new ArrayList<String>();
    
    private List<String> siteSameIp = new ArrayList<String>();
    
    private ServiceConfigModel config = null;
    
    public UpdateDomain(DomainModel domain) {
    	
    	this.config = ConfigSystem.getServiceConfigs();
    	
        this.domain = domain;
        //Load content from urls
        loadUrlDomain();
        
        loadUrlWhois();
        
        loadUrlIpAddress();
        
        loadUrlAlexa();
        
        loadUrlGoogleRank();
        
        loadUrlAgeDomain();
        
        loadUrlValidator();
        
        loadUrlGoogleService();
        
        loadUrlDns();
        
        //Get content form url loaded
        fDomainService();
        
        getWhois();
        
        getIpAddress();
        
        fAlexaService();
        
        getGooglePageRank();
        
        getAgeDomain();
        
        getValidator();
        
        getGoogleService();
        
        getDns();
    }
    
    public DomainModel getDomain() {
        return domain;
    }

    public TreeMap<String, String> getCheck() {
        return check;
    }

    public List<String> getGoogleAnalyticSites() {
        return googleAnalyticSites;
    }

    public List<String> getGoogleAdsenceSites() {
        return googleAdsenceSites;
    }

    public List<String> getSiteSameIp() {
        return siteSameIp;
    }

	/************************************* Load URL *************************************/
    private void loadUrlDomain() {
        try {
            URL urlDomain = new URL("http://" + domain.getDomain());
            fDomain = fetcher.fetchAsync(urlDomain);
        } catch (Exception e) {
            check.put("fDomain", "Can't run fDomain");
        }
    }
    
    private void loadUrlWhois() {
        try {
            URL urlWhois = new URL(config.getUrlWhois()+ UrlHelper.getMainDomain(domain.getDomain()));
            fWhois = fetcher.fetchAsync(urlWhois);
        } catch (Exception e) {
            check.put("fWhois", "Can't run fWhois");
        }
    }
    
    private void loadUrlIpAddress() {
        try {
            URL urlIpAddress = new URL(config.getUrlIpAddress() + domain.getDomain());
            fIpAddress = fetcher.fetchAsync(urlIpAddress);
        } catch (Exception e) {
            check.put("fIpAddress", "Can't run fIpAddress");
        }
    }
    
    private void loadUrlAlexa() {
        try {
            URL urlAlexa = new URL(config.getUrlAlexa() + domain.getDomain());
            fAlexa = fetcher.fetchAsync(urlAlexa);
        } catch (Exception e) {
            check.put("fAlexa", "Can't run fAlexa");
        }
    }
    
    private void loadUrlGoogleRank() {
        try {
            URL urlGoogleRank = new URL(config.getUrlGoogleRank() + domain.getDomain());
            fGoogleRank = fetcher.fetchAsync(urlGoogleRank);
        } catch (Exception e) {
            check.put("fGoogleRank", "Can't run fGoogleRank");
        }
    }
    
    private void loadUrlAgeDomain() {
        try {
            URL urlAgeDomain = new URL(config.getUrlAgeDomain());
            HTTPRequest request = new HTTPRequest(urlAgeDomain, HTTPMethod.POST);
            request.setHeader(new HTTPHeader("Content-Type", "application/x-www-form-urlencoded"));
            String params = "query=" + domain.getDomain() + "&output=cdt";
            request.setPayload(params.getBytes());
            fAgeDomain = fetcher.fetchAsync(request);

        } catch (Exception e) {
            check.put("fAgeDomain", "Can't run fAgeDomain");
        }
    }
    
    private void loadUrlValidator() {
        try {
            URL urlValidator = new URL(config.getUrlValidator() + domain.getDomain());
            fValidator = fetcher.fetchAsync(urlValidator);
            
        } catch (Exception e) {
            check.put("fValidator", "Can't run fValidator");
        }
    }
    
    private void loadUrlGoogleService() {
        try {
            URL urlGoogleService = new URL(config.getUrlGoogleService() + domain.getDomain());
            fGoogleService = fetcher.fetchAsync(urlGoogleService);
            
        } catch (Exception e) {
            check.put("fGoogleService", "Can't run fGoogleService");
        }
    }
    
    private void loadUrlDns() {
    	try {
            URL urlDns = new URL(config.getUrlDns() + domain.getDomain());
            fDns = fetcher.fetchAsync(urlDns);
            
        } catch (Exception e) {
            check.put("fDns", "Can't run fDns");
        }
    }
    
    private void loadUrlWhoisIp() {
    	try {
            URL urlWhoisIp = new URL(config.getUrlWhoisIp() + domain.getIpAddress() + "/");
            fWhoisIp = fetcher.fetchAsync(urlWhoisIp);
            
        } catch (Exception e) {
            check.put("fWhoisIp", "Can't run fWhoisIp");
        }
    }
    
    /************************************* Get content *************************************/
    private void fDomainService() {
        try {
            HTTPResponse response = (HTTPResponse) fDomain.get();
            
            // if redirects are followed, this returns the final URL we are redirected to
            URL finalUrl = response.getFinalUrl();
            this.content = this.getContentCharset(response);
            
            getServerInfo(response);
            getSiteInfo();
            getGoogleAnalytic();
            getGoogleAdsense();
            getJavascriptLibrary();
            getInlink();
            getHeading();
            getTextOnHtml();
        } catch (Exception e) {
            check.put("fDomain", "Can't response fDomain");
        }
    }
    
    private void fAlexaService() {
        try {
            HTTPResponse response = (HTTPResponse) fAlexa.get();
            
            this.alexaContent = this.getContentCharset(response);
            
            getAlexaRank();
            getDailyVisitor();
            getCountryVisitor();
            
        } catch (Exception e) {
            check.put("fAlexa", "Can't response fAlexa");
        }
    }
    
    private void getServerInfo(HTTPResponse response) {
        try {
            //Get webServer
            String webServer = "";
            String programmingLanguage = "";
            for (int i = 0; i < response.getHeaders().size(); i++) {
                if (response.getHeaders().get(i).getName().equalsIgnoreCase("server")) {
                    webServer = response.getHeaders().get(i).getValue();
                }
                
                if (response.getHeaders().get(i).getName().equalsIgnoreCase("x-powered-by")) {
                    programmingLanguage = response.getHeaders().get(i).getValue();
                }
                //System.out.println(response.getHeaders().get(i).getName() + " - " + response.getHeaders().get(i).getValue());
            }
            
            this.domain.setWebServer(webServer);
            this.domain.setProgrammingLanguage(programmingLanguage);
            
        } catch (Exception e) {
            this.check.put("server", domain.getDomain() + " - Webserver not found");
        }
    }
    
    private void getSiteInfo() {
        try {
            Document doc = Jsoup.parse(this.content);
            
            String title = "";
            String description = "";
            String keywords  = "";
            
            try {
                title = doc.select("title").first().text();
            } catch (Exception e) {
                this.check.put("title", "Can't get info title");
            }

            try {
                keywords = doc.select("meta[name=keywords]").first().attr("content");
            } catch (Exception e) {
                try {
                    keywords = doc.select("meta[http-equiv=keywords]").first().attr("content");
                } catch (Exception e2) {
                    this.check.put("keywords", "Can't get info keywords");
                }        
            }
            
            try {
                description = doc.select("meta[name=description]").first().attr("content");
            } catch (Exception e) {
                try {
                    description = doc.select("meta[http-equiv=description]").first().attr("content");
                } catch (Exception e2) {
                    if(doc.select("body").first().text().length() > 260)
                    {
                        description = doc.select("body").first().text().substring(0,250).replaceAll("[ ]+[\\w\\W]{0,10}+$", "")+" ...";
                    } else {
                        description = doc.select("body").first().text();
                    }
                }
            }
            if(description.length()>260)
            {
                description = description.substring(0,250);
            }
            
            this.domain.setTitle(title);
            this.domain.setKeywords(keywords);
            this.domain.setDescription(description);
            
        } catch (Exception e) {
            this.check.put("siteinfo", "Can't connect to domain " + domain.getDomain());
        }
    }

    private void getGoogleAnalytic() {
        int begin = content.indexOf("_gaq.push(['_setAccount', '");
        String analyticAccount = "";
        String googleAnalyticPropertyId = "";
        if(begin>0)
        {
            begin+="_gaq.push(['_setAccount', '".length();
            int end =  content.indexOf("'",begin);
            analyticAccount = content.substring(begin, end);
            googleAnalyticPropertyId = content.substring(begin, end);
            begin=analyticAccount.lastIndexOf("-")-1;
            analyticAccount = analyticAccount.substring(0, begin+1);
        }
        else
        {
             begin = content.indexOf("_gaq.push([\"_setAccount\", \"");
            if(begin>0)
            {
                begin+="_gaq.push([\"_setAccount\", \"".length();
                int end =  content.indexOf("\"",begin);
                analyticAccount = content.substring(begin, end);
                googleAnalyticPropertyId = content.substring(begin, end);
            } else {
                begin = content.indexOf("_uacct");
                if(begin > 0)
                {
                    begin+="_uacct".length();
                    analyticAccount = content.substring(begin,begin+30).replaceAll("^[ =\\'\\\"]+", "").replaceAll("[^a-zA-Z0-9\\-\\_]+[\\w\\W]+", "");
                    googleAnalyticPropertyId = analyticAccount;
                    begin=analyticAccount.lastIndexOf("-");
                    
                    analyticAccount = analyticAccount.substring(0, begin);
                } else {
                    Pattern p = Pattern.compile("UA-[0-9]+");

                    Matcher m = p.matcher(content);
                    while (m.find()) {
                        analyticAccount = m.group();
                        googleAnalyticPropertyId = analyticAccount;
                    }
                }
            }
        }
        
        this.domain.setGoogleAnalyticId(analyticAccount);
        this.domain.setGoogleAnalyticPropertyId(googleAnalyticPropertyId);
    }
    
    private void getGoogleAdsense() {
        int begin = content.indexOf("google_ad_client = \"");
        String adsAccount = "";
        if( begin > 0 )
        {
            begin+="google_ad_client = \"', '".length();
            int end =  content.indexOf("\"",begin);
            adsAccount = content.substring(begin, end);
        }
        
        this.domain.setGoogleAdsenseId(adsAccount);
    }
    
    private void getJavascriptLibrary() {
        List<String> jsLib = new ArrayList<String>();
        if(content.indexOf("jquery")>0)
            jsLib.add("jquery");
        
        if(content.indexOf("prototype.js")>0)
            jsLib.add("prototype");
        
        if(content.indexOf("extjs")>0)
            jsLib.add("extjs");
        
        if(content.indexOf("yui")>0)
            jsLib.add("yui");
        
        if(content.indexOf("dojo")>0)
            jsLib.add("dojo");
        
        if(content.indexOf("gwt")>0)
            jsLib.add("gwt");

        if(content.indexOf("easeljs")>0)
            jsLib.add("easeljs");
        
        this.domain.setJavascriptLibrary(StringHelper.listToString(jsLib, ", "));
    }
    
    private void getWhois() {
        try {
            HTTPResponse response = (HTTPResponse) fWhois.get();
            /*Document doc = Jsoup.parse(this.getContentCharset(response));
            String whois = doc.select("#content .standard .standardWhoIs table td #ctl00_cphContent_ctlWhoisInformation_standardWhoIs").html();
            if (!whois.isEmpty()) {
            	this.domain.setWhois(whois);
            }*/
            this.domain.setWhois(this.getContentCharset(response));
        } catch (Exception e) {
            check.put("fWhois", "Error when get value whois");
        }
    }
    
    private void getIpAddress() {
        try {
            HTTPResponse response = (HTTPResponse) fIpAddress.get();
            this.domain.setIpAddress(this.getContentCharset(response));
            
            loadUrlWhoisIp();
            getWhoisIp();
        } catch (Exception e) {
            check.put("ipAddress", "Can't get ipAddress for this domain " + domain.getDomain());
        }
    }
    
    private void getAlexaRank() {
        try {
            Document doc = Jsoup.parse(this.alexaContent);
            String strAlexaRank = doc.select("#siteStats").select("tbody tr td div").first().text().trim().replace(",","");
            int alexaRank = 0;
            
            try {
                if(strAlexaRank != null && !strAlexaRank.isEmpty())
                {
                    alexaRank=Integer.parseInt( strAlexaRank );
                }
            } catch (Exception e) {
                check.put("alexaGlobeRank", "Can't get position alexa rank of domain " + domain.getDomain());
            }
            
            this.domain.setAlexaRank(alexaRank);
            
        } catch (Exception e) {
            check.put("alexaRank", "Can't get position alexa rank of domain " + domain.getDomain());
        }
    }
    
    private void getGooglePageRank() {
        try {
            HTTPResponse response = (HTTPResponse) fGoogleRank.get();
            
            Document doc = Jsoup.parse(this.getContentCharset(response));
            int googlePageRank = 0;
            String google = doc.select(".show").select("span").first().select("font").text().replaceAll("[^0-9]", "");
            try {
                if(google!= null && !google.isEmpty())
                {
                    googlePageRank = Integer.parseInt(google);
                }
            } catch (Exception e) {
                check.put("googlePageRank", "Can't get position google page rank of domain " + domain.getDomain());
            }
            
            this.domain.setGooglePageRank(googlePageRank);
            
        } catch (Exception e) {
            check.put("googlePageRank", "Can't get position google page rank of domain " + domain.getDomain());
        }
    }
    
    private void getAgeDomain() {
        try {
            HTTPResponse response = (HTTPResponse) fAgeDomain.get();
            
            Document doc = Jsoup.parse(this.getContentCharset(response));
            String age = doc.select("div[class=KonaBody]").select("p").text();
               String[] ageSplit = age.split("(approximately)");

               if(ageSplit.length ==2)
               {
                   this.domain.setAge(ageSplit[1].replaceAll("[):]+", ""));
               }
            
        } catch (Exception e) {
            check.put("age", "Can't get age of domain " + domain.getDomain());
        }
    }
    
    private void getValidator() {
        try {
            HTTPResponse response = (HTTPResponse) fValidator.get();
            
            Document doc = Jsoup.parse(this.getContentCharset(response));
            
            String invalid = doc.select("#results_container .header .invalid").text();
            
            String[] str = invalid.split(",");
            
            if (str.length > 0) {
                this.domain.setValidError(Integer.parseInt(str[0].replaceAll("[\\D]*", "")));
            }
            
            if (str.length > 1) {
                this.domain.setValidWarnings(Integer.parseInt(str[1].replaceAll("[\\D]*", "")));
            }
        } catch (Exception e) {
            check.put("validator", "Can't get validator of domain " + domain.getDomain());
        }
    }
    
    private void getDailyVisitor() {
        try {
            Document doc = Jsoup.parse(this.alexaContent);
            String strVisitor = doc.select("#reach .visitors_percent").select("tr td.avg").first().text().replaceAll("[^0-9.]*", "");
            int dailyVisitor = 0;
            
            try {
                if(strVisitor != null && !strVisitor.isEmpty())
                {
                    dailyVisitor = (int) Math.round(((350 * Float.parseFloat(strVisitor)) * 100) / 0.03);
                }
            } catch (Exception e) {
                check.put("dailyVisitor", "Can't get daily visitor of domain " + domain.getDomain());
            }
            
            this.domain.setDailyVisitor(dailyVisitor);
            
        } catch (Exception e) {
            check.put("dailyVisitor", "Can't get daily visitor of domain " + domain.getDomain());
        }
    }
    
    private void getInlink() {
        try {
            Document doc = Jsoup.parse(this.content);
            Elements link = doc.select("a");
            
            ArrayList<String> listLink = new ArrayList<String>();
            String check = null;
            for(int i=0;i<link.size();i++)
            {
                check = link.get(i).attr("rel"); 
                if(Pattern.matches("^http[\\w\\W]+", link.get(i).attr("href")))
                {
                    if( check != null && !check.equals("nofollow") || check == null)
                    {
                        Pattern p = Pattern.compile("^http[^\\/]+[\\/]{2}[^\\/]+");
    
                        Matcher m = p.matcher(link.get(i).attr("href"));
                        while (m.find()) {
                            if(!listLink.contains(m.group()))
                                listLink.add(m.group());
                        }
                    }
                }
            }
            
            this.domain.setInlink(StringHelper.listToString(listLink, ","));
            
        } catch (Exception e) {
            check.put("inlink", "Can't get inlink of domain " + domain.getDomain());
        }
    }
    
    private void getHeading() {
    	try {
            Document doc = Jsoup.parse(this.content);
            Elements h1 = doc.select("H1");
            Elements h2 = doc.select("H2");
            Elements h3 = doc.select("H3");
            Elements h4 = doc.select("H4");
            Elements h5 = doc.select("H5");
            Elements h6 = doc.select("H6");
            
            String heading = "H1:" + Integer.toString(h1.size()) + ",H2:" + Integer.toString(h2.size())
            				+"H3:" + Integer.toString(h3.size()) + ",H4:" + Integer.toString(h4.size())
            				+"H5:" + Integer.toString(h5.size()) + ",H6:" + Integer.toString(h6.size());
            this.domain.setHeading(heading);
            
        } catch (Exception e) {
            check.put("heading", "Can't get heading of domain " + domain.getDomain());
        }
    }
    
    private void getTextOnHtml() {
    	try {
			int textLength = this.content.length();
			int htmlLength = this.content.replaceAll("\\<.*?>","").length();
			double d = (double) htmlLength/textLength;
			this.domain.setRatioTextHtml(1-d);
		} catch (Exception e) {
			check.put("ratioTextHtml", "Can't get ratioTextHtml of domain " + domain.getDomain());
		}
    }
    
    private void getGoogleService() {
        try {
            HTTPResponse response = (HTTPResponse) fGoogleService.get();
            Document doc = Jsoup.parse(this.getContentCharset(response));
            
            Elements elms = doc.select(".content .re_list ul.detail .w_left");
            
            for (int i = 0; i < elms.size(); i++) {
                if (elms.get(i).text().contains("Site's same Google Analytic") && elms.get(i).text().contains(this.domain.getGoogleAnalyticId())) {
                    Elements analyticLinks = elms.get(i).parent().select(".d_right a");
                    for (int j = 0; j < analyticLinks.size(); j++) {
                        String txt = analyticLinks.get(j).text().replaceAll("[^0-9a-zA-Z.]*", "");
                        googleAnalyticSites.add(txt);
                    }
                }
                
                if (elms.get(i).text().contains("Site's same Google Adsense")) {
                    Elements AdsenceLinks = elms.get(i).parent().select(".d_right a");
                    for (int j = 0; j < AdsenceLinks.size(); j++) {
                        String txt = AdsenceLinks.get(j).text().replaceAll("[^0-9a-zA-Z.]*", "");
                        googleAdsenceSites.add(txt);
                    }
                }
                
                if (elms.get(i).text().contains("IP Address")) {
                    Elements AdsenceLinks = elms.get(i).parent().select(".d_right a");
                    for (int j = 0; j < AdsenceLinks.size(); j++) {
                        String txt = AdsenceLinks.get(j).text().replaceAll("[^0-9a-zA-Z.]*", "");
                        siteSameIp.add(txt);
                    }
                } 
            }
            
        } catch (Exception e) {
            check.put("googleService", "Can't get google adsence and google analytic of domain " + domain.getDomain());
        }
    }
    
    private void getCountryVisitor() {
        try {
            Document doc = Jsoup.parse(this.alexaContent);
            Elements countries = doc.select("#visitors-by-country .tr1");
            
            List<String> visitor = new ArrayList<String>();
            for (Element country : countries) {
                try {
                    String name = country.select(".tc1").get(0).select("a").html().replaceAll("<[^>]+/>", "").replace("&nbsp;", "").trim();
                    String percent = country.select(".tc1").get(1).html().replace("%", "");
                    visitor.add(name+"-"+percent);
                } catch (Exception e) {
                }
            }
            
            this.domain.setCountryVisitor(StringHelper.listToString(visitor, ","));
        } catch (Exception e) {
            check.put("countryVisitor", "Can't get visitor by country of domain " + domain.getDomain());
        }
    }
    
    private void getDns() {
    	try {
    		HTTPResponse response = (HTTPResponse) fDns.get();
            Document doc = Jsoup.parse(this.getContentCharset(response));
            if (doc.select("#content #nameservers") != null) {
            	String nameServers = "<table>" + doc.select("#content #nameservers").html() + "</table>";
            	this.domain.setNameServers(nameServers);
            }
            
            if (doc.select("#content #soa_record") != null) {
            	String soaRecord = "<table>" + doc.select("#content #soa_record").html() + "</table>";
            	this.domain.setSoaRecord(soaRecord);
            }
            
            if (doc.select("#content #records") != null) {
            	String dnsRecords = "<table>" + doc.select("#content #records").html() + "</table>";
            	this.domain.setDnsRecords(dnsRecords);
            }
            
		} catch (Exception e) {
			check.put("dns", "Can't get dns info for domain " + domain.getDomain());
		}
    }
    
    private void getWhoisIp() {
    	try {
			HTTPResponse response = (HTTPResponse) fWhoisIp.get();
			Document doc = Jsoup.parse(this.getContentCharset(response));
			if (doc.select(".page > table tr:eq(4) table td.wrap") != null) {
				doc.select(".page > table tr:eq(4) table td.wrap > div").remove();
				String whoisIp = doc.select(".page > table tr:eq(4) table td.wrap").html();
				this.domain.setWhoisIp(whoisIp);
			}
			
		} catch (Exception e) {
			check.put("whoisIp", "Can't get whois ip info for domain " + domain.getIpAddress());
		}
    }
    
    
    private String getContentCharset(HTTPResponse response) {
        String charset = "UTF-8";
        String content = "";
        try {
			content = new String(response.getContent(),charset);
			Document doc = Jsoup.parse(content);
	        
	        int index = doc.select("meta").toString().toLowerCase().indexOf("charset=");
	        if (index != -1) {
	        	charset = doc.select("meta").toString().substring(index + 8).replaceAll("[^\\'\\\"0-9a-zA-Z\\-\\_]+[\\w\\W]+", "").replaceAll("[\\'\\\"]+", "");
	        }
	        content = new String(response.getContent(),charset);
	        
		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			//e.printStackTrace();
		}
        
    	return content;
    }
}
